package tech.abing.spider.task.utils;

import java.util.ArrayList;
import java.util.List;

import tech.abing.spider.task.model.CloseEntity;
import tech.abing.spider.util.CsvUtil;
import tech.abing.spider.util.DateUtils;
import tech.abing.spider.util.RedisUtil;
import tech.abing.spider.util.SysUtil;

/**
 * 任务结束后，获取redis中的日志信息，输出到文件
 * 
 * @Description: TODO
 * @author Ivan 2862099249@qq.com
 * @date 2015年9月6日 下午2:01:50
 * @version V1.0
 */
public class TaskLog {

	public static void main(String[] args) {

		RedisUtil redisUtil = RedisUtil.getInstance();

		String closeTask = SysUtil.getValue("redis.close_product_id");

		CsvUtil csv = new CsvUtil(CloseEntity.class, "data/crawl/closelog_"+DateUtils.getDate("yyyyMMdd")+".csv");

		List<CloseEntity> dataList = new ArrayList<CloseEntity>();

		int leaveNum = redisUtil.size(closeTask);

		int flushSize = 100;
		
		int i = 0;
		while (leaveNum > 0) {

			System.out.println("还剩："+leaveNum);
			CloseEntity closeEntity = (CloseEntity) redisUtil.rpopObj(closeTask);

			dataList.add(closeEntity);

			leaveNum = redisUtil.size(closeTask);
			
			if(++i % flushSize == 0){
				csv.setCsvData(dataList);
				csv.flush();
				
				dataList = new ArrayList<CloseEntity>();
			}
		}
		
		csv.setCsvData(dataList);
		csv.dispose();

	}

}
